package com.ruoyi.web.controller.jsoup;

import com.ruoyi.RuoYiApplication;
import com.ruoyi.common.utils.SpringContextUtil;
import com.ruoyi.common.utils.StringUtils;
import com.ruoyi.system.domain.spider.Route;
import com.ruoyi.system.mapper.RouteMapper;
import com.ruoyi.system.service.IRouteService;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.Date;

public class SpiderDemo {

    public static final String BASEURL = "http://www.jinmalvyou.com/Search/index/view_type/1/keyword/%E5%9B%BD%E5%86%85";

    private static IRouteService routeService;

    public static void main(String[] args) throws IOException {

        SpringApplication.run(RuoYiApplication.class, args);
        ApplicationContext applicationContext = SpringContextUtil.getApplicationContext();
        routeService = applicationContext.getBean(IRouteService.class);

        // Step1 根据初试地址获取分页界面
        Document document = Jsoup.connect(BASEURL).get();

        // Step2 获取分页条数据
        Elements divElements = document.select(".page>.page-block>div");

        // Step3 判断分页条数据是否存在
        if (null != divElements && divElements.size() > 0) {
            Element divElement = divElements.get(0);
            Elements children = divElement.children();
            if (null != children && children.size() > 0) {
                // Step4 如果分页条数据存在，递归爬取页面内容
                grepPageHtml(BASEURL, 1);
            }
        }
    }

    /**
     * 递归抓取html的方法
     *
     * @param url
     * @param curPage
     */
    private static void grepPageHtml(String url, int curPage) throws IOException {

        try {
            // Step4.1 通过url获取document
            Document document = Jsoup.connect(url).get();

            // Step4.2 抓取文本内容 保存到数据库
            Elements liElements = document.select(".result-list2>.rl-b2>li");
            for (Element element : liElements) {
                Route route = new Route();
                route.setCid(5); // 路线分类ID
                route.setSid(1); // 商家id,旅行社id
                route.setCount(0); // 收藏数量,默认为0
                route.setIsThemeTour("0"); // 是否是主题游 1-是 0-否

                String priceStr = element.select(".pro-price>.price>strong").first().text();
                route.setPrice(Double.valueOf(priceStr)); // 路线价格（元）

                SimpleDateFormat formate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                route.setRdate(formate.format(new Date())); // 路线抓取日期

                route.setRflag("1"); // 路线是否有效， 1-有效 0-无效
                String rimage = element.select(".pro-img img").first().attr("src");
                rimage.replace("//img.jinmalvyou.com/", "");
                route.setRimage(rimage); // 图片地址

                String rname = element.select(".pro-title>a").first().text();

                route.setRname(rname);  // 线路名称

                // 判断线路介绍是否存在
                if(StringUtils.isNotEmpty(element.select(".pro-colomn"))){
                    String routeIntroduce = element.select(".pro-colomn").first().text();
                    route.setRouteIntroduce(routeIntroduce);  // 线路介绍
                }

                String href = element.select(".pro-img>a").first().attr("href");
                String[] urlAttr = href.split("=");
                route.setSourceId(urlAttr[1]);

                routeService.insertRoute(route);

                // Step4.3 抓取图片内容 保存到指定目录
                // Step4.3.1 获取图片地址
                String imgUrl = "http:" + rimage;

                CloseableHttpClient httpClient = HttpClients.createDefault();   // 获取客户端对象，相当于浏览器
                HttpGet httpGet = new HttpGet(imgUrl);
                CloseableHttpResponse response = httpClient.execute(httpGet);
                if (response.getStatusLine().getStatusCode() == 200) {
                    // Step4.3.2 根据地址获取图片输入流
                    InputStream in = response.getEntity().getContent();
                    // Step4.3.3 创建文件输出流
                    String imgName = imgUrl.substring(imgUrl.lastIndexOf("/") + 1);
                    FileOutputStream fos = new FileOutputStream("F:\\img" + imgName);
                    // Step4.3.4 将输入流内容通过输出流输出
                    IOUtils.copy(in, fos);
                    // Step4.3.5 关闭流
                    fos.close();
                    in.close();
                }
            }

            // Step4.4 获取下一页数据,如果有，继续调用本方法:url-下一页地址,
            if (curPage == 1) {
                // Step4.4.1 判断当前页是否为第一页，是的话，下一页就是第一个a标签
                Element element = document.select(".page>.page-block a").first();
                if (null != element && StringUtils.isNotEmpty(element.attr("href"))) {
                    String nextUrl = "http://www.jinmalvyou.com" + element.attr("href");
                    curPage++;
                    grepPageHtml(nextUrl, curPage);
                }
            } else {
                // Step4.4.2 如果不是，下一页就是获取第(curPage+1)个a标签
                Element element = document.select(".page>.page-block a").get(curPage++);
                if (null != element && StringUtils.isNotEmpty(element.attr("href"))) {
                    // 如果元素存在，并且有url,在获取url继续抓取
                    String nextUrl = "http://www.jinmalvyou.com" + element.attr("href");
                    curPage++;
                    grepPageHtml(nextUrl, curPage);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (NumberFormatException e) {
            e.printStackTrace();
        } catch (UnsupportedOperationException e) {
            e.printStackTrace();
        }


    }
}


























